{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import os \n",
    "from os.path import join"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['multi_class_cross_roberta',\n",
       " 'multi_class_cross1',\n",
       " 'BertSentiEntity_cross',\n",
       " 'senti_entity_goodremove']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_dir = r'/home/njuciairs/wangshuai/competitions/finacial_models_round2'\n",
    "models = os.listdir(model_dir)\n",
    "models\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/home/njuciairs/wangshuai/competitions/finacial_models_round2/multi_class_cross1\u001b[00m\r\n",
      "├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573026942.AIRS.13972.0\r\n",
      "│   │       ├── events.out.tfevents.1573196075.AIRS.15210.27\r\n",
      "│   │       └── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573026950.AIRS.13972.1\r\n",
      "│   ├── \u001b[01;34m10\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573180302.AIRS.15210.24\r\n",
      "│   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573180305.AIRS.15210.25\r\n",
      "│   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573181396.AIRS.15210.26\r\n",
      "│   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573027124.AIRS.15210.0\r\n",
      "│   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573027132.AIRS.15210.1\r\n",
      "│   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573028203.AIRS.15210.2\r\n",
      "│   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573043806.AIRS.15210.3\r\n",
      "│   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573043812.AIRS.15210.4\r\n",
      "│   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573044886.AIRS.15210.5\r\n",
      "│   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573057365.AIRS.15210.6\r\n",
      "│   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573057369.AIRS.15210.7\r\n",
      "│   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573058449.AIRS.15210.8\r\n",
      "│   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573086938.AIRS.15210.9\r\n",
      "│   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573086942.AIRS.15210.10\r\n",
      "│   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573088023.AIRS.15210.11\r\n",
      "│   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573100391.AIRS.15210.12\r\n",
      "│   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573100395.AIRS.15210.13\r\n",
      "│   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573101479.AIRS.15210.14\r\n",
      "│   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573118523.AIRS.15210.15\r\n",
      "│   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573118530.AIRS.15210.16\r\n",
      "│   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573119596.AIRS.15210.17\r\n",
      "│   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573135028.AIRS.15210.18\r\n",
      "│   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573135031.AIRS.15210.19\r\n",
      "│   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573136118.AIRS.15210.20\r\n",
      "│   └── \u001b[01;34m9\u001b[00m\r\n",
      "│       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│           ├── events.out.tfevents.1573157183.AIRS.15210.21\r\n",
      "│           ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│           │   └── events.out.tfevents.1573157188.AIRS.15210.22\r\n",
      "│           └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│               └── events.out.tfevents.1573158267.AIRS.15210.23\r\n",
      "├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   └── \u001b[01;34mdata\u001b[00m\r\n",
      "├── \u001b[01;34mversion10\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.896667_epoch1_time20191108104956\r\n",
      "│   ├── score0.915417_epoch2_time20191108110807\r\n",
      "│   └── score0.918750_epoch4_time20191108114352\r\n",
      "├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.897208_epoch1_time20191106161643\r\n",
      "│   ├── score0.907783_epoch2_time20191106163431\r\n",
      "│   └── score0.908206_epoch5_time20191106172749\r\n",
      "├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.892901_epoch1_time20191106205446\r\n",
      "│   └── score0.921625_epoch2_time20191106211303\r\n",
      "├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.885356_epoch1_time20191107004049\r\n",
      "│   ├── score0.893724_epoch2_time20191107005846\r\n",
      "│   ├── score0.899582_epoch3_time20191107011642\r\n",
      "│   ├── score0.902092_epoch7_time20191107022822\r\n",
      "│   ├── score0.902510_epoch15_time20191107045129\r\n",
      "│   └── score0.903347_epoch17_time20191107052714\r\n",
      "├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.907900_epoch1_time20191107085343\r\n",
      "│   └── score0.928776_epoch2_time20191107091135\r\n",
      "├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.889805_epoch1_time20191107123759\r\n",
      "│   ├── score0.898482_epoch2_time20191107125559\r\n",
      "│   ├── score0.912364_epoch3_time20191107131359\r\n",
      "│   ├── score0.913232_epoch4_time20191107133155\r\n",
      "│   └── score0.920174_epoch6_time20191107140826\r\n",
      "├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.910628_epoch1_time20191107173956\r\n",
      "│   ├── score0.916667_epoch2_time20191107175737\r\n",
      "│   └── score0.923510_epoch5_time20191107185117\r\n",
      "├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.917435_epoch1_time20191107221518\r\n",
      "│   ├── score0.924901_epoch2_time20191107223322\r\n",
      "│   ├── score0.927975_epoch5_time20191107232738\r\n",
      "│   ├── score0.929293_epoch6_time20191107234536\r\n",
      "│   ├── score0.929732_epoch8_time20191108002134\r\n",
      "│   ├── score0.933245_epoch9_time20191108003930\r\n",
      "│   └── score0.933685_epoch10_time20191108005729\r\n",
      "└── \u001b[01;34mversion9\u001b[00m\r\n",
      "    ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   └── raw_rs.csv\r\n",
      "    ├── score0.895479_epoch1_time20191108042427\r\n",
      "    ├── score0.909996_epoch2_time20191108044224\r\n",
      "    ├── score0.916217_epoch3_time20191108050017\r\n",
      "    └── score0.917047_epoch11_time20191108072318\r\n",
      "\r\n",
      "60 directories, 74 files\r\n"
     ]
    }
   ],
   "source": [
    "model_name = join(model_dir,'multi_class_cross1')\n",
    "!tree $model_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/home/njuciairs/wangshuai/competitions/finacial_models_round2/multi_class_cross_roberta\u001b[00m\r\n",
      "├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573273936.AIRS.2268.0\r\n",
      "│   │       ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573273948.AIRS.2268.1\r\n",
      "│   │       ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573276124.AIRS.2268.4\r\n",
      "│   │       ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573276124.AIRS.2268.3\r\n",
      "│   │       └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573275036.AIRS.2268.2\r\n",
      "│   └── \u001b[01;34m2\u001b[00m\r\n",
      "│       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│           ├── events.out.tfevents.1573293805.AIRS.2268.5\r\n",
      "│           ├── \u001b[01;34mlosses_classification_loss\u001b[00m\r\n",
      "│           │   └── events.out.tfevents.1573293808.AIRS.2268.6\r\n",
      "│           ├── \u001b[01;34mnegative_try_desperate_try\u001b[00m\r\n",
      "│           │   └── events.out.tfevents.1573295970.AIRS.2268.9\r\n",
      "│           ├── \u001b[01;34mnegative_try_negative_try\u001b[00m\r\n",
      "│           │   └── events.out.tfevents.1573295970.AIRS.2268.8\r\n",
      "│           └── \u001b[01;34mvalidate_acc_score\u001b[00m\r\n",
      "│               └── events.out.tfevents.1573294890.AIRS.2268.7\r\n",
      "├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.896362_epoch1_time20191109125036\r\n",
      "│   ├── score0.915398_epoch2_time20191109130841\r\n",
      "│   ├── score0.918359_epoch6_time20191109143202\r\n",
      "│   └── score0.920051_epoch7_time20191109145100\r\n",
      "└── \u001b[01;34mversion2\u001b[00m\r\n",
      "    ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    ├── score0.898236_epoch1_time20191109182130\r\n",
      "    ├── score0.904801_epoch2_time20191109183926\r\n",
      "    ├── score0.915060_epoch3_time20191109190243\r\n",
      "    ├── score0.919163_epoch4_time20191109192350\r\n",
      "    ├── score0.919573_epoch6_time20191109200308\r\n",
      "    └── score0.921215_epoch7_time20191109202158\r\n",
      "\r\n",
      "17 directories, 21 files\r\n"
     ]
    }
   ],
   "source": [
    "model_name = join(model_dir,'multi_class_cross_roberta')\n",
    "!tree $model_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/home/njuciairs/wangshuai/competitions/finacial_models_round2/senti_entity_goodremove\u001b[00m\r\n",
      "├── \u001b[01;34mboard\u001b[00m\r\n",
      "│   ├── \u001b[01;34m1\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573213491.AIRS.30894.0\r\n",
      "│   │       ├── events.out.tfevents.1573219507.AIRS.30894.6\r\n",
      "│   │       ├── events.out.tfevents.1573219518.AIRS.30894.7\r\n",
      "│   │       ├── events.out.tfevents.1573224890.AIRS.30894.14\r\n",
      "│   │       ├── events.out.tfevents.1573233218.AIRS.30894.22\r\n",
      "│   │       ├── events.out.tfevents.1573239879.AIRS.30894.30\r\n",
      "│   │       ├── events.out.tfevents.1573247885.AIRS.30894.38\r\n",
      "│   │       ├── events.out.tfevents.1573255073.AIRS.30894.46\r\n",
      "│   │       ├── events.out.tfevents.1573259572.AIRS.30894.54\r\n",
      "│   │       ├── events.out.tfevents.1573266749.AIRS.30894.62\r\n",
      "│   │       ├── events.out.tfevents.1573272065.AIRS.30894.70\r\n",
      "│   │       ├── events.out.tfevents.1573272254.AIRS.30894.72\r\n",
      "│   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573213505.AIRS.30894.2\r\n",
      "│   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573213505.AIRS.30894.1\r\n",
      "│   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573213778.AIRS.30894.4\r\n",
      "│   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573213778.AIRS.30894.3\r\n",
      "│   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573213778.AIRS.30894.5\r\n",
      "│   ├── \u001b[01;34m2\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573219700.AIRS.30894.8\r\n",
      "│   │       ├── events.out.tfevents.1573224902.AIRS.30894.15\r\n",
      "│   │       ├── events.out.tfevents.1573272277.AIRS.30894.73\r\n",
      "│   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573219704.AIRS.30894.10\r\n",
      "│   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573219704.AIRS.30894.9\r\n",
      "│   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573219980.AIRS.30894.12\r\n",
      "│   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573219980.AIRS.30894.11\r\n",
      "│   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573219980.AIRS.30894.13\r\n",
      "│   ├── \u001b[01;34m3\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573225083.AIRS.30894.16\r\n",
      "│   │       ├── events.out.tfevents.1573233222.AIRS.30894.23\r\n",
      "│   │       ├── events.out.tfevents.1573272299.AIRS.30894.74\r\n",
      "│   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573225088.AIRS.30894.18\r\n",
      "│   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573225088.AIRS.30894.17\r\n",
      "│   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573225364.AIRS.30894.20\r\n",
      "│   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573225364.AIRS.30894.19\r\n",
      "│   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573225364.AIRS.30894.21\r\n",
      "│   ├── \u001b[01;34m4\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573233398.AIRS.30894.24\r\n",
      "│   │       ├── events.out.tfevents.1573239884.AIRS.30894.31\r\n",
      "│   │       ├── events.out.tfevents.1573272322.AIRS.30894.75\r\n",
      "│   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573233402.AIRS.30894.26\r\n",
      "│   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573233402.AIRS.30894.25\r\n",
      "│   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573233675.AIRS.30894.28\r\n",
      "│   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573233675.AIRS.30894.27\r\n",
      "│   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573233675.AIRS.30894.29\r\n",
      "│   ├── \u001b[01;34m5\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573240060.AIRS.30894.32\r\n",
      "│   │       ├── events.out.tfevents.1573247889.AIRS.30894.39\r\n",
      "│   │       ├── events.out.tfevents.1573272345.AIRS.30894.76\r\n",
      "│   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573240064.AIRS.30894.34\r\n",
      "│   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573240064.AIRS.30894.33\r\n",
      "│   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573240336.AIRS.30894.36\r\n",
      "│   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573240336.AIRS.30894.35\r\n",
      "│   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573240336.AIRS.30894.37\r\n",
      "│   ├── \u001b[01;34m6\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573248064.AIRS.30894.40\r\n",
      "│   │       ├── events.out.tfevents.1573255076.AIRS.30894.47\r\n",
      "│   │       ├── events.out.tfevents.1573272368.AIRS.30894.77\r\n",
      "│   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573248068.AIRS.30894.42\r\n",
      "│   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573248068.AIRS.30894.41\r\n",
      "│   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573248341.AIRS.30894.44\r\n",
      "│   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573248341.AIRS.30894.43\r\n",
      "│   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573248341.AIRS.30894.45\r\n",
      "│   ├── \u001b[01;34m7\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573255252.AIRS.30894.48\r\n",
      "│   │       ├── events.out.tfevents.1573259575.AIRS.30894.55\r\n",
      "│   │       ├── events.out.tfevents.1573272383.AIRS.30894.78\r\n",
      "│   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573255256.AIRS.30894.50\r\n",
      "│   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573255256.AIRS.30894.49\r\n",
      "│   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573255528.AIRS.30894.52\r\n",
      "│   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573255528.AIRS.30894.51\r\n",
      "│   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573255528.AIRS.30894.53\r\n",
      "│   ├── \u001b[01;34m8\u001b[00m\r\n",
      "│   │   └── \u001b[01;34mboard\u001b[00m\r\n",
      "│   │       ├── events.out.tfevents.1573259751.AIRS.30894.56\r\n",
      "│   │       ├── events.out.tfevents.1573266753.AIRS.30894.63\r\n",
      "│   │       ├── events.out.tfevents.1573272397.AIRS.30894.79\r\n",
      "│   │       ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573259756.AIRS.30894.58\r\n",
      "│   │       ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573259755.AIRS.30894.57\r\n",
      "│   │       ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573260028.AIRS.30894.60\r\n",
      "│   │       ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│   │       │   └── events.out.tfevents.1573260028.AIRS.30894.59\r\n",
      "│   │       └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│   │           └── events.out.tfevents.1573260028.AIRS.30894.61\r\n",
      "│   └── \u001b[01;34m9\u001b[00m\r\n",
      "│       └── \u001b[01;34mboard\u001b[00m\r\n",
      "│           ├── events.out.tfevents.1573266930.AIRS.30894.64\r\n",
      "│           ├── events.out.tfevents.1573272069.AIRS.30894.71\r\n",
      "│           ├── events.out.tfevents.1573272411.AIRS.30894.80\r\n",
      "│           ├── \u001b[01;34mlosses_entity_loss\u001b[00m\r\n",
      "│           │   └── events.out.tfevents.1573266936.AIRS.30894.66\r\n",
      "│           ├── \u001b[01;34mlosses_senti_loss\u001b[00m\r\n",
      "│           │   └── events.out.tfevents.1573266936.AIRS.30894.65\r\n",
      "│           ├── \u001b[01;34mvalidate_entity_f1\u001b[00m\r\n",
      "│           │   └── events.out.tfevents.1573267208.AIRS.30894.68\r\n",
      "│           ├── \u001b[01;34mvalidate_sentiment_f1\u001b[00m\r\n",
      "│           │   └── events.out.tfevents.1573267208.AIRS.30894.67\r\n",
      "│           └── \u001b[01;34mvalidate_total_score\u001b[00m\r\n",
      "│               └── events.out.tfevents.1573267208.AIRS.30894.69\r\n",
      "├── \u001b[01;34mversion1\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.912800_epoch1_time20191108194938\r\n",
      "│   ├── score0.918623_epoch2_time20191108195414\r\n",
      "│   ├── score0.921826_epoch3_time20191108195848\r\n",
      "│   ├── score0.925822_epoch4_time20191108200322\r\n",
      "│   ├── score0.935708_epoch5_time20191108200756\r\n",
      "│   └── score0.938979_epoch12_time20191108203942\r\n",
      "├── \u001b[01;34mversion2\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.937622_epoch1_time20191108213300\r\n",
      "│   ├── score0.937791_epoch2_time20191108213736\r\n",
      "│   ├── score0.963447_epoch3_time20191108214208\r\n",
      "│   └── score0.965252_epoch9_time20191108220927\r\n",
      "├── \u001b[01;34mversion3\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.934688_epoch1_time20191108230244\r\n",
      "│   ├── score0.946955_epoch3_time20191108231153\r\n",
      "│   ├── score0.947022_epoch5_time20191108232100\r\n",
      "│   ├── score0.949501_epoch6_time20191108232533\r\n",
      "│   ├── score0.951745_epoch7_time20191108233007\r\n",
      "│   ├── score0.951920_epoch9_time20191108233913\r\n",
      "│   ├── score0.954940_epoch11_time20191108234818\r\n",
      "│   ├── score0.955669_epoch18_time20191109001944\r\n",
      "│   ├── score0.956425_epoch24_time20191109004640\r\n",
      "│   └── score0.956794_epoch28_time20191109010437\r\n",
      "├── \u001b[01;34mversion4\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.926588_epoch1_time20191109012115\r\n",
      "│   ├── score0.946610_epoch2_time20191109012547\r\n",
      "│   ├── score0.952689_epoch4_time20191109013449\r\n",
      "│   ├── score0.952887_epoch5_time20191109013920\r\n",
      "│   └── score0.953727_epoch14_time20191109021947\r\n",
      "├── \u001b[01;34mversion5\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.943846_epoch1_time20191109031216\r\n",
      "│   ├── score0.948366_epoch3_time20191109032119\r\n",
      "│   ├── score0.949487_epoch6_time20191109033451\r\n",
      "│   ├── score0.951594_epoch11_time20191109035720\r\n",
      "│   └── score0.953866_epoch19_time20191109043314\r\n",
      "├── \u001b[01;34mversion6\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.915662_epoch1_time20191109052541\r\n",
      "│   ├── score0.934965_epoch2_time20191109053012\r\n",
      "│   ├── score0.943984_epoch4_time20191109053914\r\n",
      "│   ├── score0.944395_epoch11_time20191109061041\r\n",
      "│   ├── score0.944487_epoch13_time20191109061940\r\n",
      "│   └── score0.944774_epoch16_time20191109063306\r\n",
      "├── \u001b[01;34mversion7\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.928061_epoch1_time20191109072528\r\n",
      "│   ├── score0.940882_epoch2_time20191109073000\r\n",
      "│   ├── score0.947346_epoch4_time20191109073901\r\n",
      "│   └── score0.953871_epoch6_time20191109074801\r\n",
      "├── \u001b[01;34mversion8\u001b[00m\r\n",
      "│   ├── \u001b[01;34mdata\u001b[00m\r\n",
      "│   │   └── raw_rs.csv\r\n",
      "│   ├── score0.907983_epoch1_time20191109084028\r\n",
      "│   ├── score0.917036_epoch2_time20191109084500\r\n",
      "│   ├── score0.918110_epoch4_time20191109085401\r\n",
      "│   ├── score0.922546_epoch5_time20191109085831\r\n",
      "│   ├── score0.925111_epoch6_time20191109090301\r\n",
      "│   └── score0.930178_epoch16_time20191109094750\r\n",
      "└── \u001b[01;34mversion9\u001b[00m\r\n",
      "    ├── \u001b[01;34mdata\u001b[00m\r\n",
      "    │   └── raw_rs.csv\r\n",
      "    ├── score0.939119_epoch1_time20191109104008\r\n",
      "    ├── score0.940803_epoch2_time20191109104440\r\n",
      "    ├── score0.951719_epoch3_time20191109104911\r\n",
      "    ├── score0.953043_epoch5_time20191109105812\r\n",
      "    └── score0.955598_epoch9_time20191109111611\r\n",
      "\r\n",
      "82 directories, 141 files\r\n"
     ]
    }
   ],
   "source": [
    "model_name = join(model_dir,'senti_entity_goodremove')\n",
    "!tree $model_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m/home/njuciairs/wangshuai/competitions/finance_features/\u001b[00m\r\n",
      "├── \u001b[01;34mBertSentiEntity_cross\u001b[00m\r\n",
      "│   ├── feature_split 1\r\n",
      "│   ├── feature_split 2\r\n",
      "│   ├── feature_split 3\r\n",
      "│   ├── feature_split 4\r\n",
      "│   ├── feature_split 5\r\n",
      "│   ├── feature_split 6\r\n",
      "│   ├── feature_split 7\r\n",
      "│   ├── feature_split 8\r\n",
      "│   ├── feature_split 9\r\n",
      "│   ├── test_features_round2_version_goodremove 1\r\n",
      "│   ├── test_features_round2_version_goodremove 2\r\n",
      "│   ├── test_features_round2_version_goodremove 3\r\n",
      "│   ├── test_features_round2_version_goodremove 4\r\n",
      "│   ├── test_features_round2_version_goodremove 5\r\n",
      "│   ├── test_features_round2_version_goodremove 6\r\n",
      "│   ├── test_features_round2_version_goodremove 7\r\n",
      "│   ├── test_features_round2_version_goodremove 8\r\n",
      "│   ├── test_features_round2_version_goodremove 9\r\n",
      "│   ├── test_features_version 1\r\n",
      "│   ├── test_features_version 2\r\n",
      "│   ├── test_features_version 3\r\n",
      "│   ├── test_features_version 4\r\n",
      "│   ├── test_features_version 5\r\n",
      "│   ├── test_features_version 6\r\n",
      "│   ├── test_features_version 7\r\n",
      "│   ├── test_features_version 8\r\n",
      "│   ├── test_features_version 9\r\n",
      "│   ├── test_features_version_goodremove 1\r\n",
      "│   ├── test_features_version_goodremove 2\r\n",
      "│   ├── test_features_version_goodremove 3\r\n",
      "│   ├── test_features_version_goodremove 4\r\n",
      "│   ├── test_features_version_goodremove 5\r\n",
      "│   ├── test_features_version_goodremove 6\r\n",
      "│   ├── test_features_version_goodremove 7\r\n",
      "│   ├── test_features_version_goodremove 8\r\n",
      "│   └── test_features_version_goodremove 9\r\n",
      "├── \u001b[01;34mmulti_class_cross1\u001b[00m\r\n",
      "│   ├── feature_split 1\r\n",
      "│   ├── feature_split 2\r\n",
      "│   ├── feature_split 3\r\n",
      "│   ├── feature_split 4\r\n",
      "│   ├── feature_split 5\r\n",
      "│   ├── feature_split 6\r\n",
      "│   ├── feature_split 7\r\n",
      "│   ├── feature_split 8\r\n",
      "│   ├── feature_split 9\r\n",
      "│   ├── test_features_round2_version 1\r\n",
      "│   ├── test_features_round2_version 2\r\n",
      "│   ├── test_features_round2_version 3\r\n",
      "│   ├── test_features_round2_version 4\r\n",
      "│   ├── test_features_round2_version 5\r\n",
      "│   ├── test_features_round2_version 6\r\n",
      "│   ├── test_features_round2_version 7\r\n",
      "│   ├── test_features_round2_version 8\r\n",
      "│   ├── test_features_round2_version 9\r\n",
      "│   ├── test_features_version 1\r\n",
      "│   ├── test_features_version 2\r\n",
      "│   ├── test_features_version 3\r\n",
      "│   ├── test_features_version 4\r\n",
      "│   ├── test_features_version 5\r\n",
      "│   ├── test_features_version 6\r\n",
      "│   ├── test_features_version 7\r\n",
      "│   ├── test_features_version 8\r\n",
      "│   └── test_features_version 9\r\n",
      "├── \u001b[01;34mmulti_class_cross_roberta\u001b[00m\r\n",
      "│   ├── feature_split 1\r\n",
      "│   ├── feature_split 2\r\n",
      "│   ├── feature_split 3\r\n",
      "│   ├── feature_split 4\r\n",
      "│   ├── feature_split 5\r\n",
      "│   ├── feature_split 6\r\n",
      "│   ├── feature_split 7\r\n",
      "│   ├── feature_split 8\r\n",
      "│   ├── feature_split 9\r\n",
      "│   ├── test_features_round2_version 1\r\n",
      "│   ├── test_features_round2_version 2\r\n",
      "│   ├── test_features_round2_version 3\r\n",
      "│   ├── test_features_round2_version 4\r\n",
      "│   ├── test_features_round2_version 5\r\n",
      "│   ├── test_features_round2_version 6\r\n",
      "│   ├── test_features_round2_version 7\r\n",
      "│   ├── test_features_round2_version 8\r\n",
      "│   ├── test_features_round2_version 9\r\n",
      "│   ├── test_features_version 1\r\n",
      "│   ├── test_features_version 2\r\n",
      "│   ├── test_features_version 3\r\n",
      "│   ├── test_features_version 4\r\n",
      "│   ├── test_features_version 5\r\n",
      "│   ├── test_features_version 6\r\n",
      "│   ├── test_features_version 7\r\n",
      "│   ├── test_features_version 8\r\n",
      "│   └── test_features_version 9\r\n",
      "├── \u001b[01;34mpredict_features\u001b[00m\r\n",
      "│   ├── mc_rb.csv\r\n",
      "│   ├── mc_rb_se.csv\r\n",
      "│   ├── mc_rb_se_test.csv\r\n",
      "│   ├── mc_rb_se_test_round2.csv\r\n",
      "│   ├── mc_rb_test.csv\r\n",
      "│   ├── mc_rb_test_round2.csv\r\n",
      "│   └── rc_rb.csv\r\n",
      "└── \u001b[01;34msenti_entity_goodremove\u001b[00m\r\n",
      "    ├── feature_split 1\r\n",
      "    ├── feature_split 2\r\n",
      "    ├── feature_split 3\r\n",
      "    ├── feature_split 4\r\n",
      "    ├── feature_split 5\r\n",
      "    ├── feature_split 6\r\n",
      "    ├── feature_split 7\r\n",
      "    ├── feature_split 8\r\n",
      "    └── feature_split 9\r\n",
      "\r\n",
      "5 directories, 106 files\r\n"
     ]
    }
   ],
   "source": [
    "!tree /home/njuciairs/wangshuai/competitions/finance_features/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
